{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from browser import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "exps = [\n",
    "    'C100_DenseBest',\n",
    "    'C100_DenseBest_Linear',\n",
    "    'C100_SparseBest',\n",
    "    'C100_SparseBest_Linear',\n",
    "]\n",
    "paths = [os.path.expanduser(\"~/nta/results/{}\".format(e)) for e in exps]\n",
    "df = load_many(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40, 62)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Experiment Name</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>epoch_test_accuracy</th>\n",
       "      <th>noise_accuracy</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>epoch_noise_accuracy</th>\n",
       "      <th>mean_accuracy</th>\n",
       "      <th>mean_accuracy_max</th>\n",
       "      <th>epoch_mean_accuracy</th>\n",
       "      <th>...</th>\n",
       "      <th>repetitions</th>\n",
       "      <th>restore_supported</th>\n",
       "      <th>stop</th>\n",
       "      <th>sync_function</th>\n",
       "      <th>test_batch_size</th>\n",
       "      <th>test_batches_in_epoch</th>\n",
       "      <th>upload_dir</th>\n",
       "      <th>use_max_pooling</th>\n",
       "      <th>weight_decay</th>\n",
       "      <th>weight_sparsity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.5866</td>\n",
       "      <td>0.6081</td>\n",
       "      <td>56</td>\n",
       "      <td>0.1426</td>\n",
       "      <td>0.2249</td>\n",
       "      <td>51</td>\n",
       "      <td>0.36460</td>\n",
       "      <td>0.40110</td>\n",
       "      <td>51</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.5894</td>\n",
       "      <td>0.5910</td>\n",
       "      <td>34</td>\n",
       "      <td>0.1533</td>\n",
       "      <td>0.2225</td>\n",
       "      <td>40</td>\n",
       "      <td>0.37135</td>\n",
       "      <td>0.39765</td>\n",
       "      <td>40</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.5736</td>\n",
       "      <td>0.6075</td>\n",
       "      <td>32</td>\n",
       "      <td>0.1573</td>\n",
       "      <td>0.2093</td>\n",
       "      <td>24</td>\n",
       "      <td>0.36545</td>\n",
       "      <td>0.39860</td>\n",
       "      <td>47</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.5413</td>\n",
       "      <td>0.6024</td>\n",
       "      <td>39</td>\n",
       "      <td>0.1015</td>\n",
       "      <td>0.2072</td>\n",
       "      <td>12</td>\n",
       "      <td>0.32140</td>\n",
       "      <td>0.38710</td>\n",
       "      <td>30</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.5781</td>\n",
       "      <td>0.6051</td>\n",
       "      <td>35</td>\n",
       "      <td>0.1633</td>\n",
       "      <td>0.1999</td>\n",
       "      <td>30</td>\n",
       "      <td>0.37070</td>\n",
       "      <td>0.39120</td>\n",
       "      <td>35</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 62 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Experiment Name  test_accuracy  test_accuracy_max  epoch_test_accuracy  \\\n",
       "0               0         0.5866             0.6081                   56   \n",
       "1               1         0.5894             0.5910                   34   \n",
       "2               2         0.5736             0.6075                   32   \n",
       "3               3         0.5413             0.6024                   39   \n",
       "4               4         0.5781             0.6051                   35   \n",
       "\n",
       "   noise_accuracy  noise_accuracy_max  epoch_noise_accuracy  mean_accuracy  \\\n",
       "0          0.1426              0.2249                    51        0.36460   \n",
       "1          0.1533              0.2225                    40        0.37135   \n",
       "2          0.1573              0.2093                    24        0.36545   \n",
       "3          0.1015              0.2072                    12        0.32140   \n",
       "4          0.1633              0.1999                    30        0.37070   \n",
       "\n",
       "   mean_accuracy_max  epoch_mean_accuracy  ...  repetitions  \\\n",
       "0            0.40110                   51  ...           10   \n",
       "1            0.39765                   40  ...           10   \n",
       "2            0.39860                   47  ...           10   \n",
       "3            0.38710                   30  ...           10   \n",
       "4            0.39120                   35  ...           10   \n",
       "\n",
       "   restore_supported         stop  \\\n",
       "0              False  {'stop': 1}   \n",
       "1              False  {'stop': 1}   \n",
       "2              False  {'stop': 1}   \n",
       "3              False  {'stop': 1}   \n",
       "4              False  {'stop': 1}   \n",
       "\n",
       "                                       sync_function test_batch_size  \\\n",
       "0  aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "1  aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "2  aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "3  aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "4  aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "\n",
       "   test_batches_in_epoch               upload_dir  use_max_pooling  \\\n",
       "0                    100  s3://lsouza/ray/results             True   \n",
       "1                    100  s3://lsouza/ray/results             True   \n",
       "2                    100  s3://lsouza/ray/results             True   \n",
       "3                    100  s3://lsouza/ray/results             True   \n",
       "4                    100  s3://lsouza/ray/results             True   \n",
       "\n",
       "   weight_decay  weight_sparsity  \n",
       "0        0.0007              1.0  \n",
       "1        0.0007              1.0  \n",
       "2        0.0007              1.0  \n",
       "3        0.0007              1.0  \n",
       "4        0.0007              1.0  \n",
       "\n",
       "[5 rows x 62 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>mean_accuracy_max</th>\n",
       "      <th>epochs</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">CIFAR100</th>\n",
       "      <th>C100_DenseBest</th>\n",
       "      <td>0.7240</td>\n",
       "      <td>0.2252</td>\n",
       "      <td>0.4684</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_DenseBest_Linear</th>\n",
       "      <td>0.6078</td>\n",
       "      <td>0.2417</td>\n",
       "      <td>0.4174</td>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseBest</th>\n",
       "      <td>0.6885</td>\n",
       "      <td>0.2926</td>\n",
       "      <td>0.4644</td>\n",
       "      <td>92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseBest_Linear</th>\n",
       "      <td>0.6913</td>\n",
       "      <td>0.2972</td>\n",
       "      <td>0.4706</td>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 test_accuracy_max  noise_accuracy_max  \\\n",
       "dataset  name                                                            \n",
       "CIFAR100 C100_DenseBest                     0.7240              0.2252   \n",
       "         C100_DenseBest_Linear              0.6078              0.2417   \n",
       "         C100_SparseBest                    0.6885              0.2926   \n",
       "         C100_SparseBest_Linear             0.6913              0.2972   \n",
       "\n",
       "                                 mean_accuracy_max  epochs  \n",
       "dataset  name                                               \n",
       "CIFAR100 C100_DenseBest                     0.4684      98  \n",
       "         C100_DenseBest_Linear              0.4174      66  \n",
       "         C100_SparseBest                    0.4644      92  \n",
       "         C100_SparseBest_Linear             0.4706      93  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df.groupby(['dataset', 'name'])['test_accuracy_max', 'noise_accuracy_max', \n",
    "                                 'mean_accuracy_max', 'epochs']\n",
    "                                 .max().round(4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>mean_accuracy_max</th>\n",
       "      <th>epochs</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">CIFAR100</th>\n",
       "      <th>C100_DenseBest</th>\n",
       "      <td>0.6132</td>\n",
       "      <td>0.2089</td>\n",
       "      <td>0.4009</td>\n",
       "      <td>55.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_DenseBest_Linear</th>\n",
       "      <td>0.5997</td>\n",
       "      <td>0.2097</td>\n",
       "      <td>0.3918</td>\n",
       "      <td>50.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseBest</th>\n",
       "      <td>0.5459</td>\n",
       "      <td>0.2600</td>\n",
       "      <td>0.3893</td>\n",
       "      <td>64.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseBest_Linear</th>\n",
       "      <td>0.5575</td>\n",
       "      <td>0.2752</td>\n",
       "      <td>0.3894</td>\n",
       "      <td>65.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 test_accuracy_max  noise_accuracy_max  \\\n",
       "dataset  name                                                            \n",
       "CIFAR100 C100_DenseBest                     0.6132              0.2089   \n",
       "         C100_DenseBest_Linear              0.5997              0.2097   \n",
       "         C100_SparseBest                    0.5459              0.2600   \n",
       "         C100_SparseBest_Linear             0.5575              0.2752   \n",
       "\n",
       "                                 mean_accuracy_max  epochs  \n",
       "dataset  name                                               \n",
       "CIFAR100 C100_DenseBest                     0.4009    55.5  \n",
       "         C100_DenseBest_Linear              0.3918    50.6  \n",
       "         C100_SparseBest                    0.3893    64.3  \n",
       "         C100_SparseBest_Linear             0.3894    65.6  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df.groupby(['dataset', 'name'])['test_accuracy_max', 'noise_accuracy_max', \n",
    "                                 'mean_accuracy_max', 'epochs']\n",
    "                                 .mean().round(4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stats(arr):\n",
    "    mean = np.mean(arr)\n",
    "    std = np.std(arr)\n",
    "    return [round(v, 4) for v in [mean-std, mean, mean+std]]\n",
    "\n",
    "tunable_params = ['linear_percent_on']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "linear_percent_on    [0.534, 0.6674, 0.8008]\n",
       "dtype: object"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CIFAR-10 SPARSE\n",
    "filters = ((df['name']=='C100_SparseBest_Linear'))\n",
    "         \n",
    "(df[filters]\n",
    "    .sort_values('mean_accuracy_max', ascending=False)[tunable_params]\n",
    "    .head(4)\n",
    "    .apply(stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "linear_percent_on    [0.4005, 0.4927, 0.5849]\n",
       "dtype: object"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CIFAR-10 SPARSE\n",
    "filters = ((df['name']=='C100_SparseBest_Linear'))\n",
    "         \n",
    "(df[filters]\n",
    "    .sort_values('mean_accuracy_max', ascending=False)[tunable_params]\n",
    "    .tail(4)\n",
    "    .apply(stats))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Search of optimal linear weight sparsity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "exps = [\n",
    "    'C100_SparseBest_Linear1_T',\n",
    "    'C100_SparseBest_Linear2_T',\n",
    "]\n",
    "\n",
    "paths = [os.path.expanduser(\"~/nta/results/{}\".format(e)) for e in exps]\n",
    "df = load_many(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12, 62)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Experiment Name</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>epoch_test_accuracy</th>\n",
       "      <th>noise_accuracy</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>epoch_noise_accuracy</th>\n",
       "      <th>mean_accuracy</th>\n",
       "      <th>mean_accuracy_max</th>\n",
       "      <th>epoch_mean_accuracy</th>\n",
       "      <th>...</th>\n",
       "      <th>repetitions</th>\n",
       "      <th>restore_supported</th>\n",
       "      <th>stop</th>\n",
       "      <th>sync_function</th>\n",
       "      <th>test_batch_size</th>\n",
       "      <th>test_batches_in_epoch</th>\n",
       "      <th>upload_dir</th>\n",
       "      <th>use_max_pooling</th>\n",
       "      <th>weight_decay</th>\n",
       "      <th>weight_sparsity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_weight_sparsity=[0.615606968378589]</td>\n",
       "      <td>0.6942</td>\n",
       "      <td>0.6968</td>\n",
       "      <td>159</td>\n",
       "      <td>0.2254</td>\n",
       "      <td>0.2602</td>\n",
       "      <td>33</td>\n",
       "      <td>0.45980</td>\n",
       "      <td>0.46735</td>\n",
       "      <td>85</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.615607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1_weight_sparsity=[0.5108914633795211]</td>\n",
       "      <td>0.6916</td>\n",
       "      <td>0.6936</td>\n",
       "      <td>141</td>\n",
       "      <td>0.2281</td>\n",
       "      <td>0.2698</td>\n",
       "      <td>30</td>\n",
       "      <td>0.45985</td>\n",
       "      <td>0.46620</td>\n",
       "      <td>89</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.510891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2_weight_sparsity=[0.7383259279890858]</td>\n",
       "      <td>0.6997</td>\n",
       "      <td>0.7012</td>\n",
       "      <td>161</td>\n",
       "      <td>0.2225</td>\n",
       "      <td>0.2661</td>\n",
       "      <td>24</td>\n",
       "      <td>0.46110</td>\n",
       "      <td>0.46405</td>\n",
       "      <td>161</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.738326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3_weight_sparsity=[0.513634818975599]</td>\n",
       "      <td>0.6932</td>\n",
       "      <td>0.6944</td>\n",
       "      <td>156</td>\n",
       "      <td>0.2268</td>\n",
       "      <td>0.2515</td>\n",
       "      <td>122</td>\n",
       "      <td>0.46000</td>\n",
       "      <td>0.46825</td>\n",
       "      <td>122</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.513635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4_weight_sparsity=[0.2995260895161448]</td>\n",
       "      <td>0.6931</td>\n",
       "      <td>0.6949</td>\n",
       "      <td>103</td>\n",
       "      <td>0.2291</td>\n",
       "      <td>0.2681</td>\n",
       "      <td>34</td>\n",
       "      <td>0.46110</td>\n",
       "      <td>0.46510</td>\n",
       "      <td>117</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.299526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5_weight_sparsity=[0.3558040583351598]</td>\n",
       "      <td>0.6996</td>\n",
       "      <td>0.7018</td>\n",
       "      <td>150</td>\n",
       "      <td>0.2295</td>\n",
       "      <td>0.2665</td>\n",
       "      <td>69</td>\n",
       "      <td>0.46455</td>\n",
       "      <td>0.46750</td>\n",
       "      <td>85</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.355804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0_weight_sparsity=[0.712460875922671, 0.712460...</td>\n",
       "      <td>0.6952</td>\n",
       "      <td>0.6967</td>\n",
       "      <td>143</td>\n",
       "      <td>0.2024</td>\n",
       "      <td>0.2671</td>\n",
       "      <td>67</td>\n",
       "      <td>0.44880</td>\n",
       "      <td>0.47050</td>\n",
       "      <td>84</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.712461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1_weight_sparsity=[0.7331810324021211, 0.73318...</td>\n",
       "      <td>0.6898</td>\n",
       "      <td>0.6931</td>\n",
       "      <td>160</td>\n",
       "      <td>0.1850</td>\n",
       "      <td>0.2418</td>\n",
       "      <td>28</td>\n",
       "      <td>0.43740</td>\n",
       "      <td>0.45330</td>\n",
       "      <td>97</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.733181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2_weight_sparsity=[0.7007782122493296, 0.70077...</td>\n",
       "      <td>0.6943</td>\n",
       "      <td>0.6959</td>\n",
       "      <td>157</td>\n",
       "      <td>0.2004</td>\n",
       "      <td>0.2406</td>\n",
       "      <td>34</td>\n",
       "      <td>0.44735</td>\n",
       "      <td>0.45670</td>\n",
       "      <td>85</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.700778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3_weight_sparsity=[0.28786972025082924, 0.2878...</td>\n",
       "      <td>0.6997</td>\n",
       "      <td>0.7015</td>\n",
       "      <td>136</td>\n",
       "      <td>0.2078</td>\n",
       "      <td>0.2798</td>\n",
       "      <td>45</td>\n",
       "      <td>0.45375</td>\n",
       "      <td>0.45885</td>\n",
       "      <td>125</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.287870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>4_weight_sparsity=[0.32698651710912197, 0.3269...</td>\n",
       "      <td>0.6964</td>\n",
       "      <td>0.7003</td>\n",
       "      <td>159</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.2522</td>\n",
       "      <td>35</td>\n",
       "      <td>0.45310</td>\n",
       "      <td>0.45990</td>\n",
       "      <td>107</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.326987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>5_weight_sparsity=[0.4413290626165175, 0.44132...</td>\n",
       "      <td>0.6968</td>\n",
       "      <td>0.7002</td>\n",
       "      <td>161</td>\n",
       "      <td>0.2487</td>\n",
       "      <td>0.2671</td>\n",
       "      <td>106</td>\n",
       "      <td>0.47275</td>\n",
       "      <td>0.47825</td>\n",
       "      <td>106</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>{'stop': 1}</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>100</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.441329</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12 rows × 62 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Experiment Name  test_accuracy  \\\n",
       "0               0_weight_sparsity=[0.615606968378589]         0.6942   \n",
       "1              1_weight_sparsity=[0.5108914633795211]         0.6916   \n",
       "2              2_weight_sparsity=[0.7383259279890858]         0.6997   \n",
       "3               3_weight_sparsity=[0.513634818975599]         0.6932   \n",
       "4              4_weight_sparsity=[0.2995260895161448]         0.6931   \n",
       "5              5_weight_sparsity=[0.3558040583351598]         0.6996   \n",
       "6   0_weight_sparsity=[0.712460875922671, 0.712460...         0.6952   \n",
       "7   1_weight_sparsity=[0.7331810324021211, 0.73318...         0.6898   \n",
       "8   2_weight_sparsity=[0.7007782122493296, 0.70077...         0.6943   \n",
       "9   3_weight_sparsity=[0.28786972025082924, 0.2878...         0.6997   \n",
       "10  4_weight_sparsity=[0.32698651710912197, 0.3269...         0.6964   \n",
       "11  5_weight_sparsity=[0.4413290626165175, 0.44132...         0.6968   \n",
       "\n",
       "    test_accuracy_max  epoch_test_accuracy  noise_accuracy  \\\n",
       "0              0.6968                  159          0.2254   \n",
       "1              0.6936                  141          0.2281   \n",
       "2              0.7012                  161          0.2225   \n",
       "3              0.6944                  156          0.2268   \n",
       "4              0.6949                  103          0.2291   \n",
       "5              0.7018                  150          0.2295   \n",
       "6              0.6967                  143          0.2024   \n",
       "7              0.6931                  160          0.1850   \n",
       "8              0.6959                  157          0.2004   \n",
       "9              0.7015                  136          0.2078   \n",
       "10             0.7003                  159          0.2098   \n",
       "11             0.7002                  161          0.2487   \n",
       "\n",
       "    noise_accuracy_max  epoch_noise_accuracy  mean_accuracy  \\\n",
       "0               0.2602                    33        0.45980   \n",
       "1               0.2698                    30        0.45985   \n",
       "2               0.2661                    24        0.46110   \n",
       "3               0.2515                   122        0.46000   \n",
       "4               0.2681                    34        0.46110   \n",
       "5               0.2665                    69        0.46455   \n",
       "6               0.2671                    67        0.44880   \n",
       "7               0.2418                    28        0.43740   \n",
       "8               0.2406                    34        0.44735   \n",
       "9               0.2798                    45        0.45375   \n",
       "10              0.2522                    35        0.45310   \n",
       "11              0.2671                   106        0.47275   \n",
       "\n",
       "    mean_accuracy_max  epoch_mean_accuracy  ...  repetitions  \\\n",
       "0             0.46735                   85  ...            6   \n",
       "1             0.46620                   89  ...            6   \n",
       "2             0.46405                  161  ...            6   \n",
       "3             0.46825                  122  ...            6   \n",
       "4             0.46510                  117  ...            6   \n",
       "5             0.46750                   85  ...            6   \n",
       "6             0.47050                   84  ...            6   \n",
       "7             0.45330                   97  ...            6   \n",
       "8             0.45670                   85  ...            6   \n",
       "9             0.45885                  125  ...            6   \n",
       "10            0.45990                  107  ...            6   \n",
       "11            0.47825                  106  ...            6   \n",
       "\n",
       "    restore_supported         stop  \\\n",
       "0               False  {'stop': 1}   \n",
       "1               False  {'stop': 1}   \n",
       "2               False  {'stop': 1}   \n",
       "3               False  {'stop': 1}   \n",
       "4               False  {'stop': 1}   \n",
       "5               False  {'stop': 1}   \n",
       "6               False  {'stop': 1}   \n",
       "7               False  {'stop': 1}   \n",
       "8               False  {'stop': 1}   \n",
       "9               False  {'stop': 1}   \n",
       "10              False  {'stop': 1}   \n",
       "11              False  {'stop': 1}   \n",
       "\n",
       "                                        sync_function test_batch_size  \\\n",
       "0   aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "1   aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "2   aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "3   aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "4   aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "5   aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "6   aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "7   aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "8   aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "9   aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "10  aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "11  aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "\n",
       "    test_batches_in_epoch               upload_dir  use_max_pooling  \\\n",
       "0                     100  s3://lsouza/ray/results             True   \n",
       "1                     100  s3://lsouza/ray/results             True   \n",
       "2                     100  s3://lsouza/ray/results             True   \n",
       "3                     100  s3://lsouza/ray/results             True   \n",
       "4                     100  s3://lsouza/ray/results             True   \n",
       "5                     100  s3://lsouza/ray/results             True   \n",
       "6                     100  s3://lsouza/ray/results             True   \n",
       "7                     100  s3://lsouza/ray/results             True   \n",
       "8                     100  s3://lsouza/ray/results             True   \n",
       "9                     100  s3://lsouza/ray/results             True   \n",
       "10                    100  s3://lsouza/ray/results             True   \n",
       "11                    100  s3://lsouza/ray/results             True   \n",
       "\n",
       "    weight_decay  weight_sparsity  \n",
       "0         0.0007         0.615607  \n",
       "1         0.0007         0.510891  \n",
       "2         0.0007         0.738326  \n",
       "3         0.0007         0.513635  \n",
       "4         0.0007         0.299526  \n",
       "5         0.0007         0.355804  \n",
       "6         0.0007         0.712461  \n",
       "7         0.0007         0.733181  \n",
       "8         0.0007         0.700778  \n",
       "9         0.0007         0.287870  \n",
       "10        0.0007         0.326987  \n",
       "11        0.0007         0.441329  \n",
       "\n",
       "[12 rows x 62 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>mean_accuracy_max</th>\n",
       "      <th>epochs</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">CIFAR100</th>\n",
       "      <th>C100_SparseBest_Linear1_T</th>\n",
       "      <td>0.7018</td>\n",
       "      <td>0.2698</td>\n",
       "      <td>0.4683</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseBest_Linear2_T</th>\n",
       "      <td>0.7015</td>\n",
       "      <td>0.2798</td>\n",
       "      <td>0.4782</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    test_accuracy_max  noise_accuracy_max  \\\n",
       "dataset  name                                                               \n",
       "CIFAR100 C100_SparseBest_Linear1_T             0.7018              0.2698   \n",
       "         C100_SparseBest_Linear2_T             0.7015              0.2798   \n",
       "\n",
       "                                    mean_accuracy_max  epochs  \n",
       "dataset  name                                                  \n",
       "CIFAR100 C100_SparseBest_Linear1_T             0.4683     164  \n",
       "         C100_SparseBest_Linear2_T             0.4782     164  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df.groupby(['dataset', 'name'])['test_accuracy_max', 'noise_accuracy_max', \n",
    "                                 'mean_accuracy_max', 'epochs']\n",
    "                                 .max().round(4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>mean_accuracy_max</th>\n",
       "      <th>epochs</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dataset</th>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">CIFAR100</th>\n",
       "      <th>C100_SparseBest_Linear1_T</th>\n",
       "      <td>0.6971</td>\n",
       "      <td>0.2637</td>\n",
       "      <td>0.4664</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C100_SparseBest_Linear2_T</th>\n",
       "      <td>0.6980</td>\n",
       "      <td>0.2581</td>\n",
       "      <td>0.4629</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    test_accuracy_max  noise_accuracy_max  \\\n",
       "dataset  name                                                               \n",
       "CIFAR100 C100_SparseBest_Linear1_T             0.6971              0.2637   \n",
       "         C100_SparseBest_Linear2_T             0.6980              0.2581   \n",
       "\n",
       "                                    mean_accuracy_max  epochs  \n",
       "dataset  name                                                  \n",
       "CIFAR100 C100_SparseBest_Linear1_T             0.4664     164  \n",
       "         C100_SparseBest_Linear2_T             0.4629     164  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(df.groupby(['dataset', 'name'])['test_accuracy_max', 'noise_accuracy_max', \n",
    "                                 'mean_accuracy_max', 'epochs']\n",
    "                                 .mean().round(4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "tunable_params = ['weight_sparsity']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>weight_sparsity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.3881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.4950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.6019</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   weight_sparsity\n",
       "3           0.3881\n",
       "5           0.4950\n",
       "0           0.6019"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CIFAR-10 SPARSE\n",
    "filters = ((df['name']=='C100_SparseBest_Linear1_T'))\n",
    "         \n",
    "(df[filters]\n",
    "    .sort_values('mean_accuracy_max', ascending=False)[tunable_params]\n",
    "    .head(3)\n",
    "    .apply(stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>weight_sparsity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.3371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.5162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.6954</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   weight_sparsity\n",
       "1           0.3371\n",
       "4           0.5162\n",
       "2           0.6954"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CIFAR-10 SPARSE\n",
    "filters = ((df['name']=='C100_SparseBest_Linear1_T'))\n",
    "         \n",
    "(df[filters]\n",
    "    .sort_values('mean_accuracy_max', ascending=False)[tunable_params]\n",
    "    .tail(3)\n",
    "    .apply(stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>weight_sparsity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.3319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.4936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.6552</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    weight_sparsity\n",
       "11           0.3319\n",
       "6            0.4936\n",
       "10           0.6552"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CIFAR-10 SPARSE\n",
    "filters = ((df['name']=='C100_SparseBest_Linear2_T'))\n",
    "         \n",
    "(df[filters]\n",
    "    .sort_values('mean_accuracy_max', ascending=False)[tunable_params]\n",
    "    .head(3)\n",
    "    .apply(stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>weight_sparsity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.3712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.5739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.7767</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   weight_sparsity\n",
       "9           0.3712\n",
       "8           0.5739\n",
       "7           0.7767"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CIFAR-10 SPARSE\n",
    "filters = ((df['name']=='C100_SparseBest_Linear2_T'))\n",
    "         \n",
    "(df[filters]\n",
    "    .sort_values('mean_accuracy_max', ascending=False)[tunable_params]\n",
    "    .tail(3)\n",
    "    .apply(stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
